#!/bin/bash
#Copyright Barry Kauler, aug 2011, bkhome.org
#110817 find all packages to be used in a Puppy build, pet and compat.
#110821 must also work with working-dir set to /root/.packages
#110908 fix o/p file for pkgs which genericname not found.
#111014 tweak support for 3.x kernel.
#111127 no longer check if host has 'printcols', 'vercmp', it is done in 'merge2out' script.
#111127 syntax checking of db files is happening too much.
#111128 checking fields 11,12,13 of db now much faster.
#111204 do not use /var/local/woof, instead 'status' directory (better for running from different save-files).
#111204 use md5sum for all checking whether to rerun this script.
#120515 build from "gentoo" binary tarballs (refer support/gentoo).
#120719 support raspbian.
#120812 db category now allows optional subcategory (for which an icons exists in /usr/local/lib/X11/mini-icons).
#121102 file DISTRO_SPECS has new variable DISTRO_DB_SUBNAME. ex: for 14.0-based slacko, DISTRO_DB_SUBNAME=slacko14
#121102 pet_packages-${DISTRO_COMPAT_VERSION} (or pet_packages-${DISTRO_FILE_PREFIX}) is now pet_packages-${DISTRO_DB_SUBNAME.
#130306 arch linux: gz now xz. maybe only 'DISTRO_COMPAT_REPOS-arch'.
#130312 add wary5 fallback.
#130313 having problem with repo-override. $PKGLOCFLD needs to be respected (ex: pet:wary5).
#130316 fixed bug, getting duplicate :$GENERICNAME: in file FINAL_PKGS.
#130320 fix disappearing pkg in FINAL_PKGS.
#130320 get rid of |puppy|wary5| |t2|8.0rc| in fallbacks searching for pet.
#130321 bring back |t2|8.0rc|

#variable PKGS_SPECS_TABLE in file DISTRO_PKGS_SPECS-* to have optional values in 5th field:
#yes|915resolution||exe,dev,doc,nls|[pet[:<repo>]] or yes|915resolution||exe,dev,doc,nls|[compat[:<repo>]]
#exs: yes|915resolution||exe,dev,doc,nls|compat yes|915resolution|915resolution|exe,dev,doc,nls|compat:core
#...'compat' means pkg from compatible-distro, 'core' is a specific repo, 3rd or 4rd field in 'Packages-mageia-1-core'
#...3rd field is now optional for both pet/compat if 5th field present.
#previously, a pet pkg is implied by empty 3rd field (and empty 5th), however now can have an optional name qualifier...
#ex: yes|915resolution|915resolution-2011*|exe,dev,doc,nls|pet
#...optional glob wildcard(s) in 3rd field, multiple parameters allowed (comma delimiters), '-' exclusion-prefix also.
#   PET pkgs: no need to specify _DEV, _DOC, _NLS, they are implied by 4th field.
#if above is unclear, please read: http://bkhome.org/blog/?viewDetailed=02414

export LANG=C #faster.

## DS_DATE and DPS_MODIFY_DATE are md5 checksums
## when these are different it means an update has to happen

if [ $1 ] ; then #process a specific pkg only (debug)
	XPKG=$1
	DEBUG=1
fi

CURDIR="`pwd`"
if [ "$CURDIR" = "/root/.packages" ];then #110821
	. /etc/DISTRO_SPECS #has DISTRO_BINARY_COMPAT, DISTRO_COMPAT_VERSION
	[ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #121102 fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
	DS_DATE="`md5sum /etc/DISTRO_SPECS | cut -f 1 -d ' '`"
	. ./DISTRO_PKGS_SPECS #has PKGS_SPECS_TABLE, FALLBACKS_COMPAT_VERSIONS
	. ./DISTRO_COMPAT_REPOS #v431 has PKG_DOCS_DISTRO_COMPAT
	DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS | cut -f 1 -d ' '`"
else
	. ./_00build.conf
	. ./_00func
	. ./DISTRO_SPECS #has DISTRO_BINARY_COMPAT, DISTRO_COMPAT_VERSION
	[ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #121102 fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
	DS_DATE="`md5sum ./DISTRO_SPECS | cut -f 1 -d ' '`"
	source_compat_repos  # ./DISTRO_COMPAT_REPOS-
	if [ -f ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ];then
		. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #has FALLBACKS_COMPAT_VERSIONS
		DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} | cut -f 1 -d ' '`"
	else
		. ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT} #has FALLBACKS_COMPAT_VERSIONS
		DPS_MODIFY_DATE="`md5sum ./DISTRO_PKGS_SPECS-${DISTRO_BINARY_COMPAT} | cut -f 1 -d ' '`"
	fi
fi

. ./DISTRO_PET_REPOS #has PET_REPOS, PACKAGELISTS_PET_ORDER
. ./PKGS_MANAGEMENT #has PKG_NAME_ALIASES
. ./WOOFMERGEVARS #111127 need to know if a cross-build.

#variable DISTRO_KERNEL_PET is in file DISTRO_SPECS...
#ex: linux_kernel-2.6.32-44-i486-w5.pet 111014 remove if-then...
KERNELVER3="`echo -n "$DISTRO_KERNEL_PET" | cut -f 2 -d '-' | cut -f 1,2,3 -d '.' | cut -f 1 -d '_'`" #ex: 2.6.32

#modify compat-distro fallback list...
if [ "`echo -n "$FALLBACKS_COMPAT_VERSIONS" | grep "$DISTRO_COMPAT_VERSION"`" != "" ];then
	FALLBACKS_COMPAT_VERSIONS="`echo -n "$FALLBACKS_COMPAT_VERSIONS" | grep -o "${DISTRO_COMPAT_VERSION}.*"`"
	#ex: 'koala jaunty intrepid' gets reduced to 'jaunty intrepid' if DISTRO_COMPAT_VERSION=jaunty
fi

REPO_DEFAULT_SUBDIR="pet_packages-${DISTRO_DB_SUBNAME}" #121102

BINARIES='deb' #get them from packages-deb.
[ "$DISTRO_BINARY_COMPAT" = "slackware" ] && BINARIES="tgz_txz" #download to packages-tgz_txz-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "slackware64" ] && BINARIES="tgz_txz" #download to packages-tgz_txz-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "debian" ] && BINARIES="deb" #download to packages-deb-${DISTRO_COMPAT_VERSION}.
[ "$DISTRO_BINARY_COMPAT" = "gentoo" ] && BINARIES="gentoo" #120515 download to packages-gentoo-gap6
[ "$DISTRO_BINARY_COMPAT" = "raspbian" ] && BINARIES="deb_raspbian" #download to packages-deb_raspbian-${DISTRO_COMPAT_VERSION}.
BINARIES="${BINARIES}-${DISTRO_COMPAT_VERSION}"

#the files that have compatible-distro pkg docs (these were downloaded by 0setup)...
PKGLISTS_COMPAT="`echo "$PKG_DOCS_DISTRO_COMPAT" | tr ' ' '\n' | cut -f 3 -d '|' | tr '\n' ' '`" #see file DISTRO_PKGS_SPECS-ubuntu
#...format ex: 'Packages-ubuntu-intrepid-main Packages-ubuntu-intrepid-universe'

CURRDIR="`pwd`"
mkdir -p status
STATUSDIR="${CURRDIR}/status"

#for efficiency, only run this script if have too... 111204 use md5sum instead of date...
KEEPGOING='no'
for ADBFILE in $PACKAGELISTS_PET_ORDER $PKGLISTS_COMPAT
do
	NEWDATE="`md5sum $ADBFILE | cut -f 1 -d ' '`"
	OLDDATE="0"
	[ -f ${STATUSDIR}/MODIFY_DATE_${ADBFILE} ] && OLDDATE="`cat ${STATUSDIR}/MODIFY_DATE_${ADBFILE}`"
	if [ "$NEWDATE" != "$OLDDATE" ];then
		KEEPGOING='yes'
	fi
	# cache the columns used during package search
	if [ "$KEEPGOING" = "yes" ] || [ ! -f $STATUSDIR/$ADBFILE-cols ];then
		printcols ./${ADBFILE} 2 3 8 9 11 12 > $STATUSDIR/$ADBFILE-cols
	fi
done

OLD_DPS_DATE="0"
[ -f ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ] && \
	OLD_DPS_DATE="`cat ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}`"

if [ "$DPS_MODIFY_DATE" != "$OLD_DPS_DATE" ];then
	KEEPGOING='yes'
	echo -n "${DPS_MODIFY_DATE}" > ${STATUSDIR}/findpkgs_DPS_MODIFY_DATE-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
fi

OLD_DS_DATE="0"
[ -f ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS ] && \
	OLD_DS_DATE="`cat ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS`"

if [ "$DS_DATE" != "$OLD_DS_DATE" ];then
	KEEPGOING='yes'
	echo -n "$DS_DATE" > ${STATUSDIR}/findpkgs_MODIFY_DATE_DISTRO_SPECS
fi
[ ! -f ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} ] && KEEPGOING='yes'
if [ "$KEEPGOING" != "yes" -a ! "$XPKG" ];then
	echo
	echo "Exiting from support/findpkgs, already created up-to-date pkg list:"
	echo " ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}"
	echo
	exit 0
fi

rm -rf /tmp/findpkgs_tmp 2>/dev/null
mkdir /tmp/findpkgs_tmp

#remove comments from PKGS_SPECS_TABLE... make sure '|' on end... get rid of old |+udev,+whatever on end...
PKGS_SPECS_TABLE="`echo "$PKGS_SPECS_TABLE" | grep -v '^#' | grep -v '^$' | tr '\t' ' ' | tr -s ' ' | tr '+' '&' | sed -e 's%|&.*%%' | tr '&' '+' | sed -e 's% #.*%%' -e 's% $%%' -e 's%$%|%' -e 's%||$%|%'`"

echo
echo "Checking syntax of PKGS_SPECS_TABLE..."
CNT=1
for ONECHK in $PKGS_SPECS_TABLE
do
	BADFLAG='no'
	DELS="${ONECHK//[^|]}" # remove all chars but "|"
	NUMDELS="${#DELS}"     # count chars / delimiters
	[  $NUMDELS -lt 3 -o $NUMDELS -gt 5 ] && BADFLAG='yes'
	# delim         yes  |cairo |cairo|exe,dev,doc,nls|
	IFS="|" read -r YESNO GENPKG ETC <<< "$ONECHK"
	case $YESNO in
		yes|no) ok=1 ;;
		*) BADFLAG='yes' ;;
	esac
	[ "$GENPKG" = "" ]  && BADFLAG='yes'
	if [ "$BADFLAG" = "yes"  ];then
		echo "NUMDELS=$NUMDELS"
		echo "YESNO=$YESNO"
		echo "SYNTAX ERROR LINE ${CNT}: '$ONECHK'"
		echo "$PKGS_SPECS_TABLE" > /tmp/pst-dump
		exit 1
	fi
	CNT=$(($CNT + 1))
done

echo -n "" > /tmp/findpkgs_tmp/pet_log1
echo -n "" > /tmp/findpkgs_tmp/compat_log1
echo -n "" > /tmp/findpkgs_tmp/PREFERRED_PKGS
echo -n "" > /tmp/findpkgs_tmp/FINAL_PKGS

# * compiledPTNS is to select a pkg that most closely matches the 
#   compile-environment of the compat-distro...
# * ||| means architecture-independent pkg, needs high preference, 
#   immediately after exact-compiled match...
petcompiledPTNS="|${DISTRO_BINARY_COMPAT}|${DISTRO_COMPAT_VERSION}|" #ex: '|ubuntu|intrepid|'
case ${DISTRO_TARGETARCH} in
	x86)	petcompiledPTNS="${petcompiledPTNS} |common32|" ;;
	x86_64)	petcompiledPTNS="${petcompiledPTNS} |common64|" ;;
esac
petcompiledPTNS="${petcompiledPTNS} |||"
for ONECOMPATVERSION in $FALLBACKS_COMPAT_VERSIONS #sliding scale, ex: koala jaunty intrepid
do
	[ "$ONECOMPATVERSION" = "$DISTRO_COMPAT_VERSION" ] && continue
	[ "$ONECOMPATVERSION" = "4" ] && break #bottom of ladder, drop down to |puppy| test only. (i have some |puppy| in common repo, want to find them before |puppy|4|)
	petcompiledPTNS="${petcompiledPTNS} |${DISTRO_BINARY_COMPAT}|${ONECOMPATVERSION}|"
done

# * need hack, older quirky pkgs compiled with |t2|8.0rc| and have some of these in common repo...
# 130320 get rid of |puppy|wary5| |t2|8.0rc| ... 130321 bring back |t2|8.0rc| ...
petcompiledPTNS="${petcompiledPTNS} |${DISTRO_BINARY_COMPAT}| |t2|8.0rc| |puppy| |" #increasingly desperate. 130312 add wary5.

#problem with ||| above high priority, compileddistro|compiledrelease| are optional fields and may be missing.
#printcols <file> 2 3 8 9 11 12 (see below) does insert 11 & 12 delimiters if missing, but that would bump the
#priority of that pkg -- results in pkgs in Packages-puppy-4-official getting chosen.
for ADBFILE in $PACKAGELISTS_PET_ORDER $PKGLISTS_COMPAT
do
	NEWSUM="`md5sum $ADBFILE | cut -f 1 -d ' '`"
	OLDSUM="0"
	[ -f ${STATUSDIR}/MODIFY_DATE_${ADBFILE} ] && OLDSUM="`cat ${STATUSDIR}/MODIFY_DATE_${ADBFILE}`"
	if [ "$NEWSUM" != "$OLDSUM" ];then
		DB_compileddistro="`echo -n "$ADBFILE" | cut -f 2 -d '-'`" #ex: puppy
		DB_compiledrelease="`echo -n "$ADBFILE" | cut -f 3 -d '-'`" #ex: 4
		DB_repo="`echo -n "$ADBFILE" | cut -f 4 -d '-'`" #ex: official
		#db fields: pkgname|nameonly|version|pkgrelease|category[;subcategory]|size|path|fullfilename|dependencies|description|compileddistro|compiledrelease|repo|
		echo "Fixing compileddistro|compiledrelease| fields in $ADBFILE"
		FIELD11PTN="s%VERTBARCHAR$%VERTBARCHAR${DB_compileddistro}VERTBARCHAR%"
		FIELD12PTN="s%VERTBARCHAR$%VERTBARCHAR${DB_compiledrelease}VERTBARCHAR%"
		sed -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e 's%|%VERTBARCHAR%' -e "${FIELD11PTN}" -e "${FIELD12PTN}" -e 's%VERTBARCHAR$%VERTBARCHARVERTBARCHAR%' -e 's%\(VERTBARCHAR[^|]*|[^|]*|$\)%\1|%'  -e 's%VERTBARCHAR%|%g' ${ADBFILE} > /tmp/findpkgs_tmp/dbfilefixed
		REALNEWSUM="`md5sum /tmp/findpkgs_tmp/dbfilefixed | cut -f 1 -d ' '`"
		if [ "$REALNEWSUM" != "$NEWSUM" ];then
			cp -f /tmp/findpkgs_tmp/dbfilefixed $ADBFILE
			NEWSUM="$REALNEWSUM"
		fi
		echo -n "$NEWSUM" > ${STATUSDIR}/MODIFY_DATE_${ADBFILE}
	fi
done


if [ "$XPKG" ] ; then
	XRESULT=$(echo "$PKGS_SPECS_TABLE" | grep "|${XPKG}|")
	[ "$XRESULT" ] && PKGS_SPECS_TABLE="$XRESULT"
fi

######################
###  FIRST SEARCH  ###
######################
echo
echo "First pass at finding packages to be used in the Puppy build..."
echo > FINDPKGS-NOT-FOUND.log

search_func() {

 SEARCHNUM="$1"
 for ONEPKGSPEC in $PKGS_SPECS_TABLE
 do
	[ "$ONEPKGSPEC" = "" ] && continue
	# delim         yes  |cmake      |cmake,cxxlibs,libarchive,libidn|exe>dev,dev,doc,nls
	IFS="|" read -r YESNO GENERICNAME BINARYPARTNAMES FOUR PKGLOCFLD ETC <<< "$ONEPKGSPEC"
	[ "$YESNO" = "no" ] && continue
	BINARYPARTNAMES="${BINARYPARTNAMES//,/ }" #convert "," to " "
	if [ "$BINARYPARTNAMES" = "" ];then
		xBINARYPARTNAMES="$GENERICNAME"
	else
		#cut out any to be excluded ( '-' prefix)... BINARYEXCLUDED0
		BINARYEXCLUDED0=""
		xBINARYPARTNAMES=""
		for BPART in ${BINARYPARTNAMES} ; do
			case $BPART in
				-*) BINARYEXCLUDED0+=" ${BPART#-}" ;; #remove leading '-'
				*) xBINARYPARTNAMES+=" ${BPART}" ;;
			esac
		done
	fi

	excludePTNS=""
	if [ "$BINARYEXCLUDED0" ];then
		BINARYEXCLUDED0="`echo "${BINARYEXCLUDED0# }" | tr ' ' '\n'`"
		#i think that dashes need to be backslashed, change blob * wildcard to reg.expr.
		excludePTNS="`echo -n "$BINARYEXCLUDED0" | sed -e 's%\\-%\\\\-%g' -e 's%\\*%.*%g' -e 's%^%|%'`"
		echo "$excludePTNS" > /tmp/findpkgs_tmp/excludePTNS
	fi
	
	PKGLOC2=''; PKGLOC1='pet'                        #original behaviour
	[ "$BINARYPARTNAMES" != "" ] && PKGLOC1='compat' #original behaviour
	#PKGLOCFLD="`echo "$ONEPKGSPEC" | cut -f 5 -d '|'`"
	if [ "$PKGLOCFLD" ];then
		#ex:                pet:tahr
		IFS=":" read -r PKGLOC1 PKGLOC2 <<< "$PKGLOCFLD"
	fi
	if [ "$PKGLOC1" = "pet" ];then
		PACKAGELISTS_ORDER="$PACKAGELISTS_PET_ORDER"
		xPACKAGELISTS_ORDER="$PACKAGELISTS_PET_ORDER"
		compiledPTNS="$petcompiledPTNS"
	else
		PACKAGELISTS_ORDER="$PKGLISTS_COMPAT"
		xPACKAGELISTS_ORDER="$PKGLISTS_COMPAT"
		compiledPTNS='.' #allow everything through.
	fi

	if [ "$PKGLOC2" ];then
		xPACKAGELISTS_ORDER=""
		for XPACKAGE_LIST in $PACKAGELISTS_ORDER ; do
			#exs: tahr (3rd field in Packages-puppy-tahr-official)
			case $XPACKAGE_LIST in *"-${PKGLOC2}-"*) xPACKAGELISTS_ORDER+="$XPACKAGE_LIST " ;; esac
		done
		if [ ! "$xPACKAGELISTS_ORDER" ];then
			for XPACKAGE_LIST in $PACKAGELISTS_ORDER ; do
				 #exs: core, nonfree, contrib, official (4th field in Packages-puppy-wary5-official)
				case $XPACKAGE_LIST in *"-${PKGLOC2}")  xPACKAGELISTS_ORDER+="$XPACKAGE_LIST " ;; esac
			done
		fi
	fi

	echo -n "$GENERICNAME "
  
	for APARTNAME in $xBINARYPARTNAMES
	do
		#APARTNAME could have glob wildcard(s)...
		case "$APARTNAME" in
			*\*) BEGIN_WITH="|" ;; #if has '*' on end, then look in full pkg name.
			*)   BEGIN_WITH="^" ;;
		esac
		namePTN="${APARTNAME//-/\\-}"      #replace "-" with "\-"
		namePTN="${namePTN//\*/.*}"        #replace "*" with ".*"
		namePTN="${BEGIN_WITH}${namePTN}|" #^namePTN| or |namePTN|
		#in wary, i have many pkgs that are _DEV only, need to find these, a hack fix to find most...
		devnamePTN="${namePTN%|}_DEV|" #remove last char "|" and add "_DEV|"

		if [ "$DEBUG" ] ; then
			echo
			echo "$namePTN"
			echo "$devnamePTN"
			echo "$compiledPTNS"
			echo "$excludePTNS"
			echo "$xPACKAGELISTS_ORDER"
		fi

		FNDSPECS=""
		xFNDSPECS=""

		for acompiledPTN in $compiledPTNS
		do
			for PKGLIST in $xPACKAGELISTS_ORDER #ex: Packages-puppy-drake-official Packages-puppy-common-official Packages-puppy-4-official
			do
				#2 3 8 9 11 12 is: nameonly version fullpkgname dependencies compileddistro compiledrelease
				FNDSPECS="`grep "$namePTN" $STATUSDIR/$PKGLIST-cols | grep "$acompiledPTN"`"
				[ "$excludePTNS" ] && [ "$FNDSPECS" ] && \
					FNDSPECS="`echo "$FNDSPECS" | grep -v -f /tmp/findpkgs_tmp/excludePTNS`"
				#in wary, i have many pkgs that are _DEV only, need to find these, a hack fix to find most...
				if [ ! "$FNDSPECS" ] ; then
					FNDSPECS="`grep "$devnamePTN" $STATUSDIR/$PKGLIST-cols | grep "$acompiledPTN"`"
					if [ "$FNDSPECS" ] ; then # if devnamePTN was found,
						# but namePTN exists with a different acompiledPTN
						if [ "`grep "$namePTN" $STATUSDIR/$PKGLIST-cols`" ] ; then
							FNDSPECS='' # do not include devnamePTN
						fi
					fi
					[ "$excludePTNS" ] && [ "$FNDSPECS" ] && \
						FNDSPECS="`echo "$FNDSPECS" | grep -v -f /tmp/findpkgs_tmp/excludePTNS`"
				fi
				if [ "$FNDSPECS" ];then
					xFNDSPECS=$(
					for ONESPEC in $FNDSPECS ; do #maybe more than one find.
						#ex: abiword|2.3.4|abiword-2.3.4.pet|+gtk+|puppy|wary5|Packages-puppy-wary5-official
						echo "${ONESPEC}${PKGLIST}"
					done)
					break 2 #pkg(s) found.
				fi
			done
		done

		if [ ! "$FNDSPECS" ] ; then
			echo
			echo "WARNING: $APARTNAME pkg was not found!" | tee -a FINDPKGS-NOT-FOUND.log
			continue
		fi

		while read ONESPEC #maybe more than one pkg found.
		do

			IFS="|" read -r F1 F2 F3 F4 F5 F6 F7 F8 ETC <<< "$ONESPEC"

			#-- second search --
			if [ "$SEARCHNUM" = "2" ];then
   				#if there is a preferred pkg (after versioned-dep checking in this script) use that instead...
				if [ "$PREFERRED_PKGS" ];then
					if [ "$PKGLOCFLD" = "" ];then #130313 see also below. need to respect repo-override, even if a preferred version logged.
						fndnamePTN="^${F1}|" #ex: ^abiword|
						PREFSPEC="`grep "$fndnamePTN" /tmp/findpkgs_tmp/PREFERRED_PKGS`"
						if [ "$PREFSPEC" ] ; then
							ONESPEC="$PREFSPEC"
							IFS="|" read -r F1 F2 F3 F4 F5 F6 F7 F8 ETC <<< "$ONESPEC"
						fi
					fi
				fi

				fndnamePTN="|${F1}|"  #ex: |abiword|
				versionPTN="|${F2}|"  #ex: |2.3.4|
				fullnamePTN="|${F3}|" #ex: |abiword-2.3.4.pet|
				#REPODBFILE="`echo -n "$ONESPEC" | grep -o '|Packages-[^|]*' | cut -f 2 -d '|'`"
				REPODBFILE="${ONESPEC##*|Packages-}"
				REPODBFILE="Packages-${REPODBFILE%%\|*}"
				#log the complete db entry, prefixed with generic-name, pet/compat and repo db filename...
				devnamePTN="|${F1}_DEV|" #ex: |abiword_DEV|
				docnamePTN="|${F1}_DOC|" #ex: |abiword_DOC|
				nlsnamePTN="|${F1}_NLS|" #ex: |abiword_NLS|

				for onePTN in $fndnamePTN $devnamePTN $docnamePTN $nlsnamePTN
				do
					FULLDBENTRY="`grep "$onePTN" $REPODBFILE | grep "$versionPTN"`"
					if [ "$FULLDBENTRY" ];then
						#130313 i think this is where having problem. a repo-override $PKGLOCFLD needs to be respected (ex: pet:wary5)...
						if [ "$PKGLOCFLD" ];then
							EXISTENTRY="`grep "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS | grep "$REPODBFILE"`"
						else
							EXISTENTRY="`grep "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS`" #previous code.
						fi
						if [ "$EXISTENTRY" = "" ];then
							#ex: :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
							echo ":${GENERICNAME}:|${PKGLOC1}|${REPODBFILE}|${FULLDBENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS
						else
							grep -v "$onePTN" /tmp/findpkgs_tmp/FINAL_PKGS > /tmp/findpkgs_tmp/FINAL_PKGS2
							mv -f /tmp/findpkgs_tmp/FINAL_PKGS2 /tmp/findpkgs_tmp/FINAL_PKGS
							#130316 in arch build getting duplicate generic names, ex:
							# :cups::cups:|pet|Packages-puppy-wary5-official|cups_DEV-1.4.8-w5c|cups_DEV|1.4.8-w5c||BuildingBlock|216K||cups_DEV-1.4.8-w5c.pet|+cups|common unix printing system|puppy|wary5||
							gnPTN1=':'"${GENERICNAME}"'[:|]'
							if [ "$(echo -n "${EXISTENTRY}" | grep "${gnPTN1}")" = "" ];then
								#ex: :audio::a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
								echo ":${GENERICNAME}:${EXISTENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS
							else #130320 fix disappearing pkg.
								#ex: :audio::a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
								echo "${EXISTENTRY}" >> /tmp/findpkgs_tmp/FINAL_PKGS
							fi
						fi
					fi
				done

			#-- 1st search --#
			else
				#field-4 is dependencies.
				VERDEPS="${F4}"
				VERDEPS="${VERDEPS//,+/ }" # convert ",+" to " "
				VERDEPS="${VERDEPS//,/ }"  # convert "," to " " (rare cases)
				VERDEPS="${VERDEPS#+}"     # remove leading "+"
				if [ "$VERDEPS" = "" ];then
					#ex: abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|
					echo "${F1}|${F2}|${F3}|${F5}|${F6}|${F7}|" >> /tmp/findpkgs_tmp/${PKGLOC1}_log1
				else
					#create one line for each dep.
					for ONEVERDEP in ${VERDEPS}
					do
						#ex: abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|aiksaurus&gt5.6.7
						echo "${F1}|${F2}|${F3}|${F5}|${F6}|${F7}|${ONEVERDEP}" >> /tmp/findpkgs_tmp/${PKGLOC1}_log1
					done
				fi
			fi

		done <<< "$xFNDSPECS"
	done
 done
} #end search_func

if [ "$USE_FINDPKGS_SEARCH_HELPER" = "yes" ] ; then
  support/findpkgs-search-helper 1 --packagelists-pet-order="$PACKAGELISTS_PET_ORDER" --pkglists-compat="$PKGLISTS_COMPAT" --pkgs-specs-table="$PKGS_SPECS_TABLE" --petcompiledPTNS="$petcompiledPTNS" --debug="$DEBUG" --verbose
else
  search_func 1 #first search.
fi
sync

#---------------------------------
#append dummy linux_kernel line...
echo "linux_kernel|${KERNELVER3}|linux_kernel-${KERNELVER3}.pet|||" >> /tmp/findpkgs_tmp/pet_log1

#don't forget, need read aliases for deps, variable PKG_NAME_ALIASES in file PKGS_MANAGEMENT...
#want to handle multiple conditions ex: aiksaurus&gt5.6.7&lt5.7.8

echo -n "" > /tmp/findpkgs_tmp/fnd_ok_deps
echo -n "" > /tmp/findpkgs_tmp/fnd_ok_apps

#handle aliases...
aliasesPTNS="`echo "$PKG_NAME_ALIASES" | tr ' ' '\n' | grep -v '^$' | sed -e 's%^%|%' -e 's%$%|%' -e 's%,%|,|%g' -e 's%\\*%.*%g'`"

if [ "$CHECK_PKG_DEPENDENCIES" = "yes" ] ; then
  echo -e "\nProcessing dependencies..."
  if [ "$USE_FINDPKGS_DEP_HELPER" = "yes" ] ; then
    if [ "$USE_PKG_NAME_ALIASES" = "yes" ] ; then
      support/findpkgs-dep-helper --pkg-name-aliases="$PKG_NAME_ALIASES" /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 --verbose
    else
      support/findpkgs-dep-helper /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 --verbose
    fi
  else
  for ONESPEC in `cat /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 | grep '&' | tr '\n' ' '`
  do
	#ex: ONESPEC=abiword|2.3.4|abiword-2.3.4-w5.pet|puppy|wary5|Packages-puppy-wary5-official|aiksaurus&gt5.6.7
	appPTN="^`echo -n "$ONESPEC" | cut -f 1 -d '|'`|" #pattern of the application name ex: ^abiword|
	appVER="`echo -n "$ONESPEC" | cut -f 2 -d '|'`" #ex: 2.3.4
	DEPFLD="`echo -n "$ONESPEC" | cut -f 7 -d '|'`"
	DEPNAME="`echo -n "$DEPFLD" | cut -f 1 -d '&'`"
	DEPCONDS="`echo -n "$DEPFLD" | cut -f 2-9 -d '&' | tr '&' ' '`"
	depnamePTN="|${DEPNAME}|"
	echo -n "$DEPNAME "
  
	# This is currently broken... I think it relied on a bug in busybox grep
	#ex: PKG_NAME_ALIASES has an entry cxxlibs,glibc*,libc-* the above creates |cxxlibs|,|glibc.*|,|libc\-.*|
#	echo "$aliasesPTNS" > /tmp/findpkgs_tmp/aliasesPTNS
#	singlePTNS="`echo "$aliasesPTNS" | tr ',' '\n'`"
#	echo "$singlePTNS" > /tmp/findpkgs_tmp/singlePTNS
#	FNDALIAS="`echo -n "$depnamePTN" | grep -f /tmp/findpkgs_tmp/singlePTNS | sed -e 's%\\.\\*%*%g' -e 's%\\\\-%-%g'`"  #translate .* \- back to * -
#	if [ "$FNDALIAS" ];then
#		aliasPTNS="`echo -n "$FNDALIAS" | grep -f /tmp/findpkgs_tmp/aliasesPTNS | tr ',' ' '`" #ex: |cxxlibs| |glibc.*| |libc\-.*|
#	else
		aliasPTNS="^${DEPNAME}|"
#	fi

	for namePTN in $aliasPTNS
	do
		for FNDPKG in `cat /tmp/findpkgs_tmp/compat_log1 /tmp/findpkgs_tmp/pet_log1 | grep "$namePTN" | tr '\n' ' '`
		do
			#found a dependency, now check its version is ok...
			FNDVER="`echo -n "$FNDPKG" | cut -f 2 -d '|'`"

			condFLG='good'
			for ACOND in $DEPCONDS #ex: gt5.6.7 lt6.7.8
			do
				DEPOP="`echo -n "$ACOND" | cut -c 1,2`"
				DEPVER="`echo -n "$ACOND" | cut -c 3-99`"
				if ! vercmp ${FNDVER} ${DEPOP} ${DEPVER};then
					condFLG='bad'
				fi
			done
			[ "$condFLG" = "bad" ] && continue
   
			#version is ok.
			#log dep info, make sure only latest version is logged...
			depVERprev="`grep "$namePTN" /tmp/findpkgs_tmp/fnd_ok_deps | cut -f 2 -d '|'`"
			if [ "$depVERprev" = "" ];then
				echo "$FNDPKG" >> /tmp/findpkgs_tmp/fnd_ok_deps
			else
				if vercmp ${FNDVER} gt ${depVERprev};then
					grep -v "$namePTN" /tmp/findpkgs_tmp/fnd_ok_deps > /tmp/findpkgs_tmp/fnd_ok_deps2
					mv -f /tmp/findpkgs_tmp/fnd_ok_deps2 /tmp/findpkgs_tmp/fnd_ok_deps
					echo "$FNDPKG" >> /tmp/findpkgs_tmp/fnd_ok_deps
				fi
			fi

			#log actual pkg info as well as deps, make sure latest version logged...
			appVERprev="`grep "$appPTN" /tmp/findpkgs_tmp/fnd_ok_apps | cut -f 2 -d '|'`"
			if [ "$appVERprev" = "" ];then
				echo "$ONESPEC" >> /tmp/findpkgs_tmp/fnd_ok_apps
			else
				if vercmp ${appVER} gt ${appVERprev};then
					grep -v "$appPTN" /tmp/findpkgs_tmp/fnd_ok_apps > /tmp/findpkgs_tmp/fnd_ok_apps2
					mv -f /tmp/findpkgs_tmp/fnd_ok_apps2 /tmp/findpkgs_tmp/fnd_ok_apps
					echo "$ONESPEC" >> /tmp/findpkgs_tmp/fnd_ok_apps
				fi
			fi
		done
	done

  done
  fi
fi

echo -n "" > /tmp/findpkgs_tmp/PREFERRED_PKGS
cat /tmp/findpkgs_tmp/fnd_ok_deps >> /tmp/findpkgs_tmp/PREFERRED_PKGS
cat /tmp/findpkgs_tmp/fnd_ok_apps >> /tmp/findpkgs_tmp/PREFERRED_PKGS

sort --key=3,3 --field-separator="|" -u /tmp/findpkgs_tmp/PREFERRED_PKGS > /tmp/findpkgs_tmp/PREFERRED_PKGS2
grep -v '^linux_kernel|' /tmp/findpkgs_tmp/PREFERRED_PKGS2 > /tmp/findpkgs_tmp/PREFERRED_PKGS #remove dummy entry.
cp -f /tmp/findpkgs_tmp/PREFERRED_PKGS ${STATUSDIR}/findpkgs_PREFERRED_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #so retained between boots.
sync

[ -s /tmp/findpkgs_tmp/PREFERRED_PKGS ] && PREFERRED_PKGS=1 #second search

#######################
###  SECOND SEARCH  ###
#######################
#now find all pkgs again, this time use prefs recorded in /tmp/findpkgs_tmp/PREFERRED_PKGS
echo
echo
echo "Second pass at finding packages to be used in the Puppy build..."
echo -e "\n-\nSECOND PASS\n-\n" >> FINDPKGS-NOT-FOUND.log

if [ "$USE_FINDPKGS_SEARCH_HELPER" = "yes" ] ; then
  support/findpkgs-search-helper 2 --packagelists-pet-order="$PACKAGELISTS_PET_ORDER" --pkglists-compat="$PKGLISTS_COMPAT" --pkgs-specs-table="$PKGS_SPECS_TABLE" --petcompiledPTNS="$petcompiledPTNS" --debug="$DEBUG" --verbose
else
  search_func 2 #result will be in /tmp/findpkgs_tmp/FINAL_PKGS
fi
#...each line is full db entry for package, prefixed with :generic-name:, pet/compat and repo-db-file
# ex: :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
# 'compat' means pkg obtained from compatible-distro, in case of Wary they are also PETs.
# :generic-name: usually coincides with actual name in package. multiple pkgs may be combined into one generic-name.
# if the same pkg is combined into two different generic-names, then the line will look like this: :gcc_lib::gcc_dev:| (ex from Slacko).

#110908 not-found pkgs are left in here without the genericname etc fields preprended...
grep '^:' /tmp/findpkgs_tmp/FINAL_PKGS > /tmp/findpkgs_tmp/FINAL_PKGSx
mv -f /tmp/findpkgs_tmp/FINAL_PKGSx /tmp/findpkgs_tmp/FINAL_PKGS

sync
sort --key=4,4 --field-separator="|" /tmp/findpkgs_tmp/FINAL_PKGS > ${STATUSDIR}/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION} #so retained between boots.
sync
echo
echo
echo "Script finished. Result is in file"
echo " status/findpkgs_FINAL_PKGS-${DISTRO_BINARY_COMPAT}-${DISTRO_COMPAT_VERSION}
Each line is full db entry for pkg, prefixed with :generic-name:, pet/compat
 and db-file. ex:
 :a52dec:|compat|Packages-puppy-wary5-official|a52dec-0.7.4-w5|a52dec|0.7.4-w5||BuildingBlock|68K||a52dec-0.7.4-w5.pet||A free ATSC A52 stream decoder|puppy|wary5||
 'compat' means pkg from compat-distro, in case of Wary they are also PETs
 :generic-name: is 2nd field in PKGS_SPECS_TABLE, note in some cases multiple
 packages are combined into same genericname, or same pkg can be shared by
 multiple genericnames -- latter case, line will look like: :gcc_lib::gcc_dev:|"

### END ###
